# -*- coding: utf-8 -*-

import codecs
import json
import  os
import  urllib2
import  MySQLdb
import MySQLdb.cursors
from twisted.enterprise import adbapi
from scrapy.pipelines.images import ImagesPipeline

class JsonWithEncodingPipeline(object):
    #将信息存为json格式
    def __init__(self):
        self.file = codecs.open('article.json', 'w', encoding="utf-8")

    def process_item(self, item, spider):
        lines = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item

    def spider_closed(self, spider):
        self.file.close()

   # def spider_closed(self, spider):
   #     self.file.close()


class ScrapyArticlePipeline(object):
    ##将信息存为txt格式
    def process_item(self, item, spider):
        with open("my_meiju.txt", 'a') as fp:
            fp.write(item['name'].encode("utf8") + '\n')


class MyImagesPipeline(object):
    #导出图片
    def __init__(self):
        self.img_index = 1


    def process_item(self, item, spider):

        #导出到当前目录image，相对路径
        currentname=os.path.dirname(__file__)
        parentname=os.path.dirname(currentname)
        imgPath= os.path.join(parentname,"image")
        #导出到d盘下的abc目录 绝对路径
        #imgPath="D:\\abc"
        if not os.path.isdir(imgPath):
            os.mkdir(imgPath)
        #for url in item["front_image_url"]:
        if 'front_image_url' in item:
            url=item["front_image_url"]
            print("下载：url")
            try:
                res=urllib2.urlopen(url)
                if str(res.code)!='200':
                    print("未下载成功：url")

            except Exception as e:
                print(e.message)
                print("下载失败：{0}",url)

            filename=os.path.join(imgPath,str(self.img_index)+'.jpg')
            with open(filename,'wb') as f:
                f.write(res.read())
                print ("下载完成\n")
                self.img_index+=1
        return  item

# 自定义图片自动下载
class ArticleImagePipeline(ImagesPipeline):
    def item_completed(self, results, item, info):
        for ok, value in results:
            image_file_path = value["path"]  # 获取文件路径
        item["front_image_path"] = image_file_path
        return item

class MysqlPipline(object):
    #采用同步的机制写入mysql
    def __init__(self):
        self.conn = MySQLdb.connect('47.93.119.63', 'scrapy', '123', 'scrapy', charset="utf8", use_unicode=True,cp_reconnect=True)
        self.cursor = self.conn.cursor()
        self.createtable()

    def process_item(self, item, spider):
        insert_sql = """
              insert into jobbole_article(name, url, create_date, fav_nums)
              VALUES (%s, %s, %s, %s)
          """
        self.cursor.execute(insert_sql, (item["name"], item["url"], item["create_date"], item["fav_nums"]))
        self.conn.commit()

    def createtable(self):

        self.cursor.execute("drop table if exists  jobbole_article")
        sql = """
                       create table jobbole_article(
                       name VARCHAR (50),
                       url VARCHAR (100),
                       create_date VARCHAR (100),
                       fav_nums VARCHAR (100) 
                       )
                       """
        self.cursor.execute(sql)

#异步操作数据库
class MysqlTwistedPipline(object):
    def __init__(self,dbpool):
        # 把dbpool接收到对象中来，数据连接池
        self.dbpool=dbpool

        #写入setting文件
    @classmethod
    def from_settings(cls,settings):

        dbparms=dict(
            host=settings["MYSQL_HOST"],
            db=settings["MYSQL_DBNAME"],
            user=settings["MYSQL_USER"],
            passwd=settings["MYSQL_PASSWORD"],
            charset='utf8',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True
        )
        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
        return cls(dbpool)
    def handle_error(self, failure):
        #处理异步插入的异常
        print (failure)

    def process_item(self,item,spider):
        # 使用twisted将mysql插入变成异步执行
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.handle_error)  # 处理异常

    def do_insert(self,cursor,item):

        #insert_sql, params = item.get_insert_sql()
        insert_sql = """
                      insert into jobbole_articleTwisted(name, url, create_date, fav_nums)
                      VALUES (%s, %s, %s, %s)
                  """
        cursor.execute(insert_sql, (item["name"], item["url"], item["create_date"], item["fav_nums"]))

        print (insert_sql)




